"""
Database server runs via HTTP
released under New BSD License
Author(s): Deepak<iapain@gmail.com>, Tomas<tomas@cactilio.com>
"""
import sys
import getopt
import urllib2_file
import urllib2, urllib
import simplejson as json
from collections import deque
from wsgiref import simple_server
from settings import *
import random
import sqlite3
import time
import thread
from cgi import parse_qsl
import cgi
import shutil
from cStringIO import StringIO


APP_NAME = 'aydabtu'
APP_VERSION = '0.1'
USER_AGENT = '%s/%s' % (APP_NAME, APP_VERSION)
MAX_RNEIGHBOURS = 2
MAX_SNEIGHBOURS = MAX_RNEIGHBOURS * 2
MAX_NEIGHBOURS = MAX_RNEIGHBOURS + MAX_SNEIGHBOURS
  

random_neighbors = []
similar_neighbors = []

port = DEFAULT_PORT
utility = DEFAULT_UTILITY 
role = "dumb"
address = DEFAULT_HOST

WRITE_IDENTIFIERS = ['insert', 'create', 'delete', 'drop', 'alter']
HTTP_FAILURE = False


class DeadNodeFoundException(Exception):
    pass

class URLError(Exception):
    pass


def is_read_only(sql):
    sql = sql.strip().lower()
    for r in WRITE_IDENTIFIERS:
        if sql.startswith(r):
            return False
    return True
    

def error(msg):
    "Renders error msg in json"
    return json.dumps({'stat':'failed', 'err':msg, 'version':APP_VERSION})

def help_command():
    "Renders help msg"
    return json.dumps({'stat':'ok', 'version':APP_VERSION, 'help':\
                       'Use /query?q=sql there is a table called "stocks"'})


def save_uploaded_file (form_field, params_post, length, upload_dir=os.path.dirname(__file__)):
    """This saves a file uploaded by an HTML form.
       The form_field is the name of the file input field from the form.
       For example, the following form_field would be "file_1":
           <input name="file_1" type="file">
       The upload_dir is the directory where the file will be written.
       If no file was uploaded or if the field does not exist then
       this does nothing.
    """
    form = params_post
    if not form.has_key(form_field): return
    fileitem = form[form_field]
    if not fileitem.file: return
    fout = open(fileitem.filename+".tmp", 'wb')
    chunksize = 65536 # 64KB
    mem = StringIO()
    while 1:
        remain = int(length) - mem.tell()
        print remain
        if remain <= 0: break
        chunk = fileitem.file.read(min(chunksize, remain))
        if not chunk: break
        mem.write(chunk)
    fout.write(mem.getvalue())
    fout.close()
    mem.close()
    shutil.copy(fileitem.filename+".tmp", fileitem.filename)
    os.unlink(fileitem.filename+".tmp")


#main database handler
def handle_database_app(environ,start_response):
    """
    Database handling logic and replication should go here
    Outputting data in JSON
    """    
    conn = sqlite3.connect(DB_FILE)
    c = conn.cursor()
    start_response('200 OK',[('Content-type','application/x-javascript')])
    query = environ.get('QUERY_STRING', '')
    client_ip = environ.get('REMOTE_ADDR', '')
    path = environ.get('PATH_INFO', '/')
    params = dict(parse_qsl(environ['QUERY_STRING']))    

    #handles home page
    if path == '/':
        print client_ip
        return json.dumps({APP_NAME:'Welcome', 'version':APP_VERSION, \
                           'role': role, 'utility':utility, \
                           'random_neighbors':random_neighbors,\
                           'similar_neighbors':similar_neighbors},indent=4)
    
    #replicate page
    if path == '/replicate':
        can_update = False
        params_post = cgi.FieldStorage(fp=environ['wsgi.input'], environ=environ)
        if not params_post.has_key('ts') or not params_post.has_key('file'):
                return error("No ts(timestamp) or file")
        try:
            fp = open('replication.log', 'r')
            lines = fp.readlines()
            fp.close()
            if float(params_post.getvalue('ts')) > float(lines[0]):
                can_update = True
        except IOError:
            print "IOError"
            can_update = True
            
            
        if can_update:
            try:
                save_uploaded_file('file', params_post, environ["CONTENT_LENGTH"])
            except Exception, e:
                print e
                return error("IOError in replication")
            fp = open('replication.log', 'w')
            fp.write(params_post.getvalue('ts'))
            fp.close()
            return json.dumps({'stat': 'ok', 'msg':'replication done'})
        else:
            return json.dumps({'stat': 'ok', 'msg':'replication discarded by client'})
        

    if path == "/senddb":
        try:
            fp = open(DB_FILE, 'rb')
            c = fp.read()
            fp.close()
            return c
        except:
            pass

    #handles help page
    if path == '/help':
        return help_command()

    #handles get_detail page
    if path == '/get-details':
        #if we are a bootstrapping node there is a possibility that we don't have
        #any neighbours. So we gather each node trying to bootstrap with us
        #and add it as a neighbour if we don't have
        #the bootstrap requester must give its port, role and utility
        if params.has_key('port'):
            random_neighbors.append({'address':'%s:%s' % (client_ip, params['port']), 'utility':params['utility'], 'role':params['role']})
        return  json.dumps({'utility':utility, 'role': role})

    if path == '/get-random-neighbours':
        return  json.dumps({'random_neighbors':random_neighbors},indent=4)

    if path == '/get-similar-neighbors':
        return  json.dumps({'similar_neighbors':similar_neighbors},indent=4)
    
    #handle sql queries
    if path == '/query':
        if not params.has_key('q'):
            return error("query is required, please pass it ?q=sql")
        
        sql = params['q'].lower()
        
        #if we are dumb we must send the query to the master or the best neighbour
        if role == "dumb" and is_read_only(sql):
            try:
                best_neighbour = get_best_neighbour(random_neighbors+similar_neighbors)
                if not params.get('f', '') == '1':
                    try:
                        remote_query = make_query(sql, best_neighbour['address'])
                    except DeadNodeFoundException:
                        raise IndexError
                    #to know where the result came from, without caring about bumps
                    if remote_query.get('source', '') == "local":
                       remote_query['source'] = best_neighbour['address']
                    return json.dumps(remote_query,indent=4)
            except IndexError:
                pass
               
        #if we are not dumb, we are master (in this beta version, later this will change)
        try:
            c.execute('%s' % params['q'])
        except Exception, e:
            return error(str(e))
        
        
        if not is_read_only(sql):
            conn.commit()
            if not params.get('f', '') == '1':#Avoid infinite loop here
                try:
                    replicate_query(sql)
                except DeadNodeFoundException:pass
            return json.dumps({'stat':'ok', 'source':'local', 'path':path, \
                               'query':params.get('q', None), 'count':0,\
                            'dataset': []}, indent=4)
        res = c.fetchall()
        desc = [item[0] for item in c.description]
        ds = [dict(zip(desc, row)) for row in res]
        return json.dumps({'stat':'ok', 'source':'local', 'query':params.get('q', None),\
                           'count':len(ds), 'path':path, 'dataset': ds},indent=4)
    
    return json.dumps({'stat':'fail', 'err':'404 Page not found'})


def delete_node(address):
    for i in random_neighbors:
        if i.has_key('address') and i['address'] == address:
            try:
                random_neighbors.remove(i)
            except ValueError:
                pass
    for i in similar_neighbors:
        if i.has_key('address') and i['address'] == address:
            try:
                random_neighbors.remove(i)
            except ValueError:
                pass
    

def ask_details(peer):
    """
    gets the utility and role of a known peer (useful for bootstrapping)
    """
    request = urllib2.Request('http://%s/get-details?port=%s&utility=%s&role=%s' % (peer,port,utility,role))
    #cool user agent can be a little security check
    request.add_header('User-Agent', USER_AGENT)
    opener = urllib2.build_opener()
    try:
        response = json.loads(opener.open(request).read())
    except URLError:
        return {}
    return response

def make_query(query, peer):
    """
    makes a sql query to a remote peer
    """
    request = urllib2.Request('http://%s/query?%s' % (peer, urllib.urlencode({'q':query, 'f':'1'})))
    print 'http://%s/query?%s' % (peer, urllib.urlencode({'q':query, 'f':'1'}))
    request.add_header('User-Agent', USER_AGENT)
    opener = urllib2.build_opener()
    try:
        response = json.loads(opener.open(request).read())
    except:#blindly delete node if exception
        delete_node(peer)
        raise DeadNodeFoundException
        return {}
    return response

#def replicate_query(query):
#    """
#    Replicate SQL of other nodes
#    """
#    try:
#        node = get_best_neighbour(random_neighbors+similar_neighbors)
#    except IndexError:
#        node = None
#    if node is None:
#        print "No node to replicate"
#        return
#    else:
#        return make_query(query, node['address'])

def replicate_query(query):
    "Replicates query to all other replicas also prevent double replication"
    cache = {}
    for i in random_neighbors:
        if i.has_key('address') and not cache.has_key(i['address']):
            try:
                make_query(query, i['address'])
                cache[i['address']] = 1
            except:
                pass
    for i in similar_neighbors:
        if i.has_key('address') and not cache.has_key(i['address']):
            try:
                make_query(query, i['address'])
                cache[i['address']] = 1
            except:
                pass

#Not in use
def make_replication(time_interval=10):
    """
    makes replication on remote node
    """
    user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
    headers = { 'User-Agent' : user_agent }
    while 1:
        try:
            node = get_best_neighbour(random_neighbors+similar_neighbors)
        except IndexError:
            node = None
            print "no replication node"
        if not node is None:
            ts = time.time()
            url = 'http://%s/replicate' % (node['address'])
            data = {'ts': str(ts), 'file':open(DB_FILE)}
            req = urllib2.Request(url, data, headers)
            try:
                response = urllib2.urlopen(req)
                print response.read()
            except Exception, e:
                print str(e)
        time.sleep(time_interval)

def ask_random_neighbors(peer):
    """
    gets the random neighbours of a peer
    """
    HTTP_FAILURE = False
    request = urllib2.Request('http://%s/get-random-neighbours' % peer)    
    request.add_header('User-Agent', USER_AGENT)
    opener = urllib2.build_opener()
    try:
        response = json.loads(opener.open(request).read())
    except Exception, e:
        HTTP_FAILURE = True
        return {}
    if HTTP_FAILURE:
        get_replica_db()
    return response['random_neighbors']

def ask_similar_neighbors(peer):
    """
    gets the simillar neighbours of a peer
    """
    HTTP_FAILURE = False
    request = urllib2.Request('http://%s/get-similar-neighbors' % peer)    
    request.add_header('User-Agent', USER_AGENT)
    opener = urllib2.build_opener()
    try:
        response = json.loads(opener.open(request).read())
    except Exception, e:
        HTTP_FAILURE = True
        return {}
    if HTTP_FAILURE:
        get_replica_db()
    return response['similar_neighbors']


def get_best_neighbour(neighbour_list):
    """
    gets the neighbour with highest utility in a neighbour list
    """
    best_candidate = neighbour_list[0]

    for n in neighbour_list[1:]:
        if n['role'] == "master":
            return n
        if n['utility'] < best_candidate['utility']:
            best_candidate = n

    return best_candidate  


def check_neighbours(time_interval=1):
    """
    Check neighbours in ever t secs and del old nodes
    """
    global random_neighbors, similar_neighbors
    while 1:
        try:
            #If the lists of neighbours are full, disconnect random neighbour
            if len(random_neighbors) + len(similar_neighbors) >= MAX_NEIGHBOURS:
                if random.randint(0,1) == 0:
                    random.shuffle(similar_neighbors)
                    del similar_neighbors[-1]
                else:
                    random.shuffle(random_neighbors)
                    del random_neighbors[-1]                
    
            #add random neighbour from a peer
            if len(random_neighbors) < MAX_RNEIGHBOURS and len(random_neighbors) > 0:            
                random.shuffle(random_neighbors)           
                neighbour = random_neighbors[0]
                gossiped_neighbours = ask_random_neighbors(neighbour['address'])
                #remove from the obtained list the nodes that are our neighbours already
                gossiped_neighbours = filter(lambda x:x not in random_neighbors, gossiped_neighbours)
                random.shuffle(gossiped_neighbours)
                if len(gossiped_neighbours) > 0:
                    #don't add ourselves to the neighbour list!                
                    if gossiped_neighbours[0]['address'] != ('%s:%s' % (address, port)):
                        random_neighbors.append(gossiped_neighbours[0])
                    elif len(gossiped_neighbours) > 1:
                        random_neighbors.append(gossiped_neighbours[1])
    
            #find most simillar neighbour from a peer and add it
            if len(similar_neighbors) < MAX_SNEIGHBOURS and len(random_neighbors) + len(similar_neighbors) > 0:
                all_neighbours = random_neighbors + similar_neighbors
                print all_neighbours
                random.shuffle(all_neighbours)            
                neighbour = all_neighbours[0]            
                gossiped_neighbours = list(ask_similar_neighbors(neighbour['address'])) + list(ask_random_neighbors(neighbour['address']))
                if len(gossiped_neighbours) > 0:
                    #remove from the obtained list the nodes that are our neighbours already
                    gossiped_neighbours = filter(lambda x:x not in similar_neighbors, gossiped_neighbours)
                    simillar = get_simillar_candidate(gossiped_neighbours)
                    if simillar:
                        found = False
                        for s in similar_neighbors:
                            if s['address'] == simillar['address']:
                                found = True
                                break
                        if not found:
                            similar_neighbors.append(simillar)
        except Exception, e:
            print e   
        
        #print "similar ->", similar_neighbors
        #print "random ->", random_neighbors
        time.sleep(time_interval)
   

def get_simillar_candidate(all_neighbours):
    similar = {}
    diff = 10000000
    for n in all_neighbours:
        print abs(int(n['utility']) - utility)
        if abs(int(n['utility']) - utility) < diff and n['address'] != '%s:%s' % (address, port):
            diff = abs(int(n['utility']) - utility)
            similar = n
    return similar


def get_replica_db():
    "Gets replication DB from master node"
    request = urllib2.Request('http://%s/senddb' % BOOTSTRAP)
    request.add_header('User-Agent', USER_AGENT)
    opener = urllib2.build_opener()
    f = opener.open(request)
    try:
        fp = open(DB_FILE, 'wb')
        for l in f:
            fp.write(l)
        fp.close()
    except IOError:
        pass

#initialization as module
def start_node(ut=utility, pt=port, rl=role):
    global utility
    global port
    global role

    utility = ut
    port = pt
    role = rl

    #very dirty bootstrapping, just for testing    
    try:
        bootstrap_details = ask_details(BOOTSTRAP)
        random_neighbors.append({'address':BOOTSTRAP, 'utility':bootstrap_details['utility'], 'role':bootstrap_details['role']})
        print random_neighbors
    except Exception, e:
        None    
                
    
    #gossiping checks in every t sec here its 60 secs
    thread.start_new(check_neighbours, (10,))
    
    #thread.start_new(make_replication, (10,))
    
    if ROLE == 'REPLICA':
        get_replica_db()
    
    
    #server init
  
    httpd = simple_server.WSGIServer(
        (DEFAULT_HOST, port),
            simple_server.WSGIRequestHandler,
        )

    #print "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA ", port

    #serve web forever
    httpd.set_app(handle_database_app)        

    try:
        print 'Use Control-C to exit'
        httpd.serve_forever()
    except KeyboardInterrupt:
        print 'Exiting'

    

## Initalization from command line

if __name__ == "__main__":
    try:                                
        opts, args = getopt.getopt(sys.argv[1:], "hp:u:", ["help", "port=", "utility="]) 
    except getopt.GetoptError:           
        print "AYDABTU parameters: --help --port [portnum] --utility [utilvalue]"    
        sys.exit(2) 
        
    for opt, arg in opts:
        if opt in ("-h", "--help"):
            print "AYDABTU parameters: --help --port [portnum] --utility [utilvalue]"
            sys.exit()
        elif opt in ("-p", "--port"):
            try:
                port = int(arg)
                DEFAULT_PORT = port
            except ValueError:
                print "Port should be a numeric value"
        elif opt in ("-u", "--utility"):
            try:
                utility = int(arg)
                if utility == 100:
                    role = "master"
            except ValueError:
                print "Utitlity should be numeric value"

    start_node(utility, port, role)


    

